#!/usr/bin/env python
# coding=utf-8

# 辽宁康辉国际旅行社 
import os, sys, re, traceback, logging
import urllib,httplib
import time
import datetime
import getopt
import urllib2
from urllib2 import URLError
from Route  import *


mainBlock_re = re.compile(r'''
        (<table[^<>]*?ID="Table4">.*?)<table[^<>]*?ID="Table5">
        ''', re.X|re.S)

vacationBlock_re = re.compile(r'''
       ((?:<tr>.*?</tr>\s*){6})
        ''', re.X|re.S)

def do_crawl():
    try:
        server_urls = [
                       ('国内旅游', 'http://www.lncct.com/travel_search_type.asp?vclass=%B9%FA%C4%DA%D3%CE&ltype=')
                       ,('出境旅游', 'http://www.lncct.com/travel_search_type.asp?vclass=%B3%F6%BE%B3%D3%CE&ltype=')
                       ,('省内旅游', 'http://www.lncct.com/travel_search_type.asp?vclass=%CA%A1%C4%DA%D3%CE&ltype=')
                       ,('国内地接', 'http://www.lncct.com/travel_search_type.asp?vclass=%B9%FA%C4%DA%B5%D8%BD%D3&ltype=')
                       ,('自由行', 'http://www.lncct.com/travel_search_type.asp?vclass=%D7%D4%D3%C9%D0%D0&ltype=')]
        
        for vacation_type, server_url in server_urls:
            html = urllib2.urlopen(server_url).read()
            html = html.decode("GB2312",'ignore').encode('utf-8')
            
            #是否继续
            #pageNums = int(re.compile(r'''页/共(\d{1,3})页''', re.X | re.S).search(html).group(1))
            goOn = True
            while goOn:
                #获取有用的块
                mainblock = mainBlock_re.findall(html)[0]
        
                #获取所有描述线路的HTML片段 
                vacationBlockList = vacationBlock_re.findall(mainblock)
                for div in vacationBlockList:
                    try:
                        route = Route()
                        #站点ID
                        route.site = 45
                        #提供商名称
                        route.supplier = '辽宁康辉国际旅行社  '
                        #供应商url
                        route.supplier_url = 'http://www.lncct.com/'
                    
                        #线路标题
                        route.title = re.compile(r'''title="([^"]*?)"''', re.X|re.S).search(div).group(1).strip()
                        #线路url
                        route.rout_url = 'http://www.lncct.com/' + re.compile(r'''href="([^"]*?)"''', re.X|re.S).search(div).group(1)
                        
                        #出发城市
                        route.outcity = '沈阳'
                        
                        #描述图片url
                        img_temp = re.compile(r'''src="(.*?)"''', re.X|re.S).search(div)
                        if img_temp:
                            route.img_url = 'http://www.lncct.com/'+img_temp.group(1)
                            
                        #行程类型
                        route.type =vacation_type                             
                        
                        #出发日期
                        route.dates = '天天发团'
   
                        #价格
                        priceTemp = re.compile(r'''<td.*?class="jiage1">\s*RMB\s*(\d+)</td>''', re.X|re.S).search(div)
                        if priceTemp:
                            route.price = priceTemp.group(1)
                        else:
                            continue
                        
                        #出发交通工具
                        route.go_t = '火车'
                        if route.title.find('飞')>=0:
                            route.go_t = '飞机'
                        if vacation_type =='省内旅游':
                            route.go_t = '旅游空调车'
                        elif vacation_type =='出境旅游':
                            route.go_t = '飞机'
                        
                        #回程交通工具
                        route.back_t = route.go_t
                                                                              
                        innerHtml = urllib2.urlopen(route.rout_url).read()
                        innerHtml = innerHtml.decode("GB2312",'ignore').encode('utf-8')

                        #进一步解析 
                        do_parse(innerHtml,route)
                    except:
                        continue
                    print route.rout_url
                    if route.days==0:
                        daysTemp = re.compile(u'''([一二三四五六七八九十\d]{1,2})(?:日|天)''', re.X|re.S).search(route.title.decode('utf-8'))
                        if daysTemp:
                            days = daysTemp.group(1)
                            if days.isdigit():
                                route.days=days
                            elif days ==u'一':
                                route.days='1'
                            elif days ==u'二':
                                route.days='2'
                            elif days ==u'三':
                                route.days='3'
                            elif days ==u'四':
                                route.days='4'
                            elif days ==u'五':
                                route.days='5'
                            elif days ==u'六':
                                route.days='6'
                            elif days ==u'七':
                                route.days='7'
                            elif days ==u'八':
                                route.days='8'
                            elif days ==u'九':
                                route.days='9'
                            elif days ==u'十':
                                route.days='10'
                    print >>sys.stderr, 'updating:', route.supplier,'， 价格：',route.price, '，交通工具:',route.go_t,'， 天数：',route.days,'， 出发日期：',route.dates,'， 出发城市：',route.outcity,'， 到达城市：',route.dest,'， title：',route.title
                    
                    route.updRoute()
                    route.updRoute_Schedules()
                    
                goOnTemp = re.compile(r'''next_page_act\.gif''', re.X | re.S).search(html)
                if(goOnTemp):
                    goOn = True
                    #print html
                    #url = 'http://www.lncct.com/travel_search_type.asp'+urllib.quote(re.compile(r'''<a\s*href="([^<>]*?)">\s*<img[^<>]*?next_page_act\.gif[^<>]*?>\s*</a>''', re.X | re.S).search(html).group(1),'utf-8')
                    url = u'http://www.lncct.com/travel_search_type.asp'+re.compile(r'''<a\s*href="([^<>]*?)">\s*<img[^<>]*?next_page_act\.gif[^<>]*?>\s*</a>''', re.X | re.S).search(html).group(1).decode('utf-8')
                    url=url.encode('GB2312')
                    #对url编码，':?=/&'不编码
                    url=urllib2.quote(url,':?=/&')
                    print '--------------------------------------------'
                    print url
                    html = urllib2.urlopen(url).read()
                    html = html.decode("GB2312",'ignore').encode('utf-8')
                else:
                    goOn = False


    except:
        print traceback.format_exc(sys.exc_info())


#解析每一天的行程安排----度假
def do_parse(html,route): 
    
    #行程
    schedule_list = re.compile(r'''(<TABLE\s*class=txt2[^<>]*?>.*?<TABLE[^<>]*?>.*?</TABLE>.*?</TABLE>)''', re.X|re.S).findall(html)
    if(schedule_list):   
            #行程天数
            route.days = len(schedule_list)
            i=0   
            for parthtml in schedule_list:
                schedule = Schedule()
        
                #第几天
                schedule.s_num = i+1
                
                #游览地点
                dest_temp = re.compile(u'''<TD[^<>]*?>.*?</TD>\s*<TD[^<>]*?>\s*([^&;()（）]+).*?</TD>''', re.X|re.S).search(parthtml.decode('utf-8'))
                if dest_temp:
                    temp = dest_temp.group(1).encode('utf-8')
                    places = re.split(r'―+|-+|/|\s',temp)
                    #游览地点
                    schedule.s_place = places[-1].replace(r'沈阳','').replace('―','')
                    schedule.s_place = re.sub('\d{1,2}:\d{1,2}','',schedule.s_place)
                    if route.dest=='':
                        #到达城市
                        route.dest = places[-1].replace(r'沈阳','').replace('―','')
                        route.dest = re.sub('\d{1,2}:\d{1,2}','',route.dest)
                
                #详细描述                
                schedule.s_brief = re.compile(r'''<TR>.*?</TR>\s*<TR>\s*<TD[^<>]*?>.*?</TD>\s*<TD[^<>]*?>\s*(.*?)\s*</TD>''', re.X|re.S).search(parthtml).group(1)
                schedule.s_brief = schedule.s_brief.replace('&nbsp;','').strip()
            
                #早餐： 1有、0没有 
                meals_temp = re.compile(r'''<SPAN[^<>]*?class=scene_in[^<>]*?>([^<>]*?)</SPAN>''', re.X|re.S).search(parthtml)
                if meals_temp:
                    meals = meals_temp.group(1)    
                    schedule.s_brf = '1' if meals.find('早')>-1  else '0'
                    schedule.s_lunch = '1' if meals.find(r'中')>-1  else '0'
                    schedule.s_dinner = '1' if meals.find(r'晚')>-1  else '0'
        
                #hotel_temp =  re.compile(r'''<SPAN[^<>]*?class=scene_in[^<>]*?>([^<>]*?)</SPAN>''', re.X|re.S).search(parthtml)   
                #酒店描述：可选       
                #if hotel_temp:
                #   schedule._hotel = hotel_temp.group(2).strip()
                route.schedules.append(schedule)
    return 

if __name__ == '__main__':
    do_crawl()
    """
    print '---------------end---------------'
    url = 'http://www.cctpage.com/travel/viewcj.asp?id=181774'
    html = urllib2.urlopen(url).read()
    html = html.decode("GB2312",'ignore').encode('utf-8')
    route = Route()
    do_parse(html,route)
    print >>sys.stderr, 'updating:', route.supplier,'， 价格：',route.price, '， 天数：',route.days,'， 出发日期：',route.dates,'， 出发城市：',route.outcity,'， 到达城市：',route.dest,'， title：',route.title
    """

